{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle  #数据存储\n",
    "import numpy as np #数据处理\n",
    "import pandas as pd #--\n",
    "import scipy.io as sio #--\n",
    "import scipy.sparse as ss #稀疏矩阵\n",
    "\n",
    "from __future__ import division\n",
    "from numpy.random import random\n",
    "from collections import defaultdict  #默认字典\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RecomSys:\n",
    "    def __init__(self):\n",
    "        #初始化\n",
    "        \n",
    "        #user and event ,index\n",
    "        self.userIndex = pickle.load(open(\"PE_userIndex.pkl\",\"rb\"))\n",
    "        self.eventIndex = pickle.load(open(\"PE_eventIndex.pkl\",\"rb\"))\n",
    "        self.userNum = len(self.userIndex)\n",
    "        self.eventNum = len(self.eventIndex)\n",
    "        \n",
    "        #矩阵R\n",
    "        self.userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "        #每个用户所参加的活动\n",
    "        self.eventForUser = pickle.load(open(\"PE_eventsForUser.pkl\",\"rb\"))\n",
    "        #每个活动参与的用户\n",
    "        self.usersForEvent = pickle.load(open(\"PE_usersForEvent.pkl\",\"rb\"))\n",
    "        \n",
    "        #由用户属性计算出的用户之间的相似度\n",
    "        #self.userSimMartix = sio.mmread(\"US_userSimMatrix\").todense()\n",
    "        self.userSimMatrix = sio.mmread(\"US_userSimMatrix\").todense()\n",
    "        \n",
    "        #y由活动属性计算出的活动之间的相似度\n",
    "        self.eventPropSim = sio.mmread(\"EV_eventPropSim\").todense()\n",
    "        self.eventContSim = sio.mmread(\"EV_eventContSim\").todense()\n",
    "        \n",
    "        #单个用户的朋友的数目\n",
    "        self.friendNum = sio.mmread(\"UF_numFriends\")\n",
    "        #用户的每个朋友参加活动的分数对该用户的影响\n",
    "        self.userFriends = sio.mmread(\"UF_userFriends\").todense()\n",
    "        #活动本身的热度\n",
    "        self.eventPopu = sio.mmread(\"EP_eventPopu\").todense()\n",
    "        \n",
    "        #基于模型的协同过滤参数初始化，训练\n",
    "        self.init_SVD()\n",
    "        self.train_SVD(trainfile = \"train.csv\")\n",
    "    ######################################################################\n",
    "    def init_SVD(self,K = 20):#初始化模型参数\n",
    "        self.K = K  \n",
    "\n",
    "        #init parameters\n",
    "        #bias\n",
    "        self.bi = np.zeros(self.eventNum)  \n",
    "        self.bu = np.zeros(self.userNum)  \n",
    "\n",
    "        #         self.userNum \n",
    "        #         self.eventNum\n",
    "\n",
    "        #the small matrix\n",
    "        self.P = random((self.userNum,self.K))/10*(np.sqrt(self.K))\n",
    "        self.Q = random((self.K, self.eventNum))/10*(np.sqrt(self.K))\n",
    "    ######################################################################\n",
    "    #def train_SVD(self,trainfile = \"train.csv\",steps = 100,gamma = 0.05,lambda = 0.15):#训练SVD模型\n",
    "    def train_SVD(self,trainfile = 'train.csv', steps=100,gamma=0.04,Lambda=0.15):\n",
    "        print(\"SVD ! The training is begining ...\")\n",
    "        ftrain = open(trainfile,\"r\")\n",
    "        ftrain.readline()#跳过第一行\n",
    "        self.mu = 0.0\n",
    "        n_records = 0  #样本计算容器\n",
    "        uids = []  #每个样本的用户索引\n",
    "        i_ids = [] #每个样本的item索引\n",
    "        R = np.zeros((self.userNum,self.eventNum))\n",
    "        \n",
    "        for line in ftrain:\n",
    "            cols = line.strip().split(\",\")\n",
    "            u = self.userIndex[cols[0]]  #用户\n",
    "            i = self.eventIndex[cols[1]] #活动\n",
    "\n",
    "            uids.append(u)\n",
    "            i_ids.append(i)\n",
    "\n",
    "            R[u,i] = int(cols[4])  #interested\n",
    "            self.mu += R[u,i]\n",
    "            n_records += 1\n",
    "\n",
    "        ftrain.close()\n",
    "        self.mu /= n_records\n",
    "\n",
    "    #\n",
    "        print (\"The SVD is trained\")\n",
    "    ######################################################################\n",
    "    def pred_SVD(self,uid,i_id):\n",
    "        #根据当前参数，预测用户uid对Item（i_id）的打分        \n",
    "        ans=self.mu + self.biasI[i_id] + self.biasU[uid] + np.dot(self.P[uid,:],self.Q[:,i_id])  \n",
    "\n",
    "        #将打分范围控制在0-1之间\n",
    "        if ans>1:  \n",
    "            return 1  \n",
    "        elif ans<0:  \n",
    "            return 0\n",
    "        return ans \n",
    "    ######################################################################\n",
    "    def sim_cal_UserCF(self, uid1, uid2 ):\n",
    "        #请补充基于用户的协同过滤中的两个用户uid1和uid2之间的相似度（根据两个用户对item打分的相似度）\n",
    "        similarity = 0.0\n",
    "        #用person相关系数\n",
    "        similarity = ssd.correlation(self.getrow(uid1).todense(),self.getrow(uid2).todense())\n",
    "        return similarity  \n",
    "    ######################################################################\n",
    "    def userCFReco(self, userId, eventId):\n",
    "        \"\"\"\n",
    "        根据User-based协同过滤，得到event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i\n",
    "          for every other user v that has a preference for i\n",
    "            compute similarity s between u and v\n",
    "            incorporate v's preference for i weighted by s into running aversge\n",
    "        return top items ranked by weighted average\n",
    "        \n",
    "        \"\"\"\n",
    "        #请补充完整代码\n",
    "        #------------------------------------------------------------------------\n",
    "        \n",
    "            #         Sim1 = 0.0\n",
    "            #         simSqe = list[]     \n",
    "            #         i = self.uesrIndex[userId]     #行索引号在usersForEvent\n",
    "            #         j = self.eventIndex[eventId]   #列索引号在usersForEvent\n",
    "            #         numUser = len(self.usersForEvent[j])  #看有多少用户参加了这个活动\n",
    "            #         for userAllCol in self.usersForEvent[j]:#遍历这个活动参与的用户\n",
    "        ans = 0.0\n",
    "        i = self.userIndex[userId]#行索引\n",
    "        j = self.eventIndex[eventId]#列索引\n",
    "\n",
    "        vs = self.userEventScores[:, j]#所有用户对此列的活动的评分特征向量\n",
    "        sims = self.userSimMatrix[i, :]#用户之间相似度特征向量\n",
    "\n",
    "        prod = sims * vs#nu*1  1*ne->nu*ne   相似度加权*对活动的打分\n",
    "\n",
    "        #         try:\n",
    "        #           return prod[0, 0] - self.userEventScores[i, j]\n",
    "        #         except IndexError:\n",
    "        #           return 0\n",
    "        \n",
    "        for x in range(0,self.userNum):#遍历 相似度加权*对活动的打分的结果矩阵prod\n",
    "            for y in range(0,self.eventNum):\n",
    "                if prod[x,y] > ans:\n",
    "                    ans = prod[x,y] #prod[x,y],顶级推荐度；\n",
    "                    #ans = self.eventIndex.columns(y)#推荐的event\n",
    "                    \n",
    "        return ans\n",
    "    ######################################################################\n",
    "    def sim_cal_ItemCF(self, i_id1, i_id2):\n",
    "        #计算Item i_id1和i_id2之间的相似性\n",
    "        #请补充完整代码\n",
    "        similarity = 0.0\n",
    "        similarity = ssd.cosine(self.getrow(i_id1).todense(),self.getrow(i_id2).todense())\n",
    "        return similarity \n",
    "    ######################################################################\n",
    "    def eventCFReco(self, userId, eventId):    \n",
    "        \"\"\"\n",
    "        根据基于物品的协同过滤，得到Event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i \n",
    "            for every item j that u has a preference for\n",
    "                compute similarity s between i and j\n",
    "                add u's preference for j weighted by s to a running average\n",
    "        return top items, ranked by weighted average\n",
    "        \"\"\"\n",
    "        #请补充完整代码\n",
    "        #------------------------------------------------------------------------\n",
    "        ans = 0.0\n",
    "\n",
    "        i = self.userIndex[userId]#行索引\n",
    "        j = self.eventIndex[eventId]#列索引\n",
    "        js = self.userEventScores[i, :]#用户对所有活动的评分矩阵的特征向量\n",
    "        psim = self.eventPropSim[:, j]#非词频活动之间的相似度特征向量\n",
    "        csim = self.eventContSim[:, j]#词频活动之间的相似度特征向量\n",
    "        pprod = js * psim#相似度加权特征向量*非词频活动之间的相似度特征向量\n",
    "        cprod = js * csim#相似度加权特征向量*词频活动之间的相似度特征向量\n",
    "\n",
    "        pscore = 0\n",
    "        cscore = 0\n",
    "        try:\n",
    "           pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "              pass\n",
    "        try:\n",
    "            cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "            pass\n",
    "        if pscore > cscore:\n",
    "           return pscore\n",
    "        else:\n",
    "            return  cscore\n",
    "                        \n",
    "        \n",
    "        \n",
    "        \n",
    "        # return ans\n",
    "    ######################################################################\n",
    "    def svdCFReco(self, userId, eventId):\n",
    "        #基于模型的协同过滤, SVD++/LFM\n",
    "        u = self.userIndex[userId]\n",
    "        i = self.eventIndex[eventId]\n",
    "\n",
    "        return self.pred_SVD(u,i)\n",
    "    ######################################################################\n",
    "    def userReco(self, userId, eventId):\n",
    "        \"\"\"\n",
    "        类似基于User-based协同过滤，只是用户之间的相似度由用户本身的属性得到，计算event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i\n",
    "          for every other user v that has a preference for i\n",
    "            compute similarity s between u and v\n",
    "            incorporate v's preference for i weighted by s into running aversge\n",
    "        return top items ranked by weighted average\n",
    "        \"\"\"\n",
    "        #------------------------------------------------------------------------\n",
    "        i = self.userIndex[userId]\n",
    "        j = self.eventIndex[eventId]\n",
    "\n",
    "        vs = self.userEventScores[:, j]\n",
    "        sims = self.userSimMatrix[i, :]\n",
    "\n",
    "        prod = sims * vs\n",
    "\n",
    "        try:\n",
    "           return prod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "           return 0\n",
    "        \n",
    "        \n",
    "        #         ans = 0.0\n",
    "        #         i = self.userIndex[userId]#行索引\n",
    "        #         j = self.eventIndex[eventId]#列索引\n",
    "\n",
    "        #         vs = self.userEventScores[:, j]#所有用户对此列的活动的评分特征向量\n",
    "        #         sims = self.userSimMatrix[i, :]#用户之间相似度特征向量\n",
    "\n",
    "        #         prod = sims * vs#nu*1  1*ne->nu*ne   相似度加权*对活动的打分\n",
    "\n",
    "\n",
    "        #         for x in range(0,self.uesrNum):#遍历 相似度加权*对活动的打分的结果矩阵prod\n",
    "        #             for y in range(0,self.eventNum):\n",
    "        #                 if prod[x,y] > ans:#prod[x,y]为event的推荐度\n",
    "        #                     #ans = prod[x,y] \n",
    "        #                     ans = self.eventIndex.columns(y)#推荐的event\n",
    "        #         return ans\n",
    "    ######################################################################\n",
    "    def eventReco(self, userId, eventId):\n",
    "        \"\"\"\n",
    "        类似基于Item-based协同过滤，只是item之间的相似度由item本身的属性得到，计算Event的推荐度\n",
    "        基本的伪代码思路如下：\n",
    "        for item i \n",
    "          for every item j that u has a preference for\n",
    "            compute similarity s between i and j\n",
    "            add u's preference for j weighted by s to a running average\n",
    "        return top items, ranked by weighted average\n",
    "        \"\"\"\n",
    "        #------------------------------------------------------------------------\n",
    "        i = self.userIndex[userId]\n",
    "        j = self.eventIndex[eventId]\n",
    "        js = self.userEventScores[i, :]\n",
    "        psim = self.eventPropSim[:, j]\n",
    "        csim = self.eventContSim[:, j]\n",
    "        pprod = js * psim\n",
    "        cprod = js * csim\n",
    "\n",
    "        pscore = 0\n",
    "        cscore = 0\n",
    "        try:\n",
    "          pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "          pass\n",
    "        try:\n",
    "          cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "        except IndexError:\n",
    "          pass\n",
    "        return pscore, cscore\n",
    "    ######################################################################\n",
    "    def userPop(self, userId):\n",
    "        \"\"\"\n",
    "        基于用户的朋友个数来推断用户的社交程度\n",
    "        主要的考量是如果用户的朋友非常多，可能会更倾向于参加各种社交活动\n",
    "        \"\"\"\n",
    "        if self.userIndex.has_key(userId):\n",
    "          i = self.userIndex[userId]\n",
    "          try:\n",
    "            return self.numFriends[0, i]\n",
    "          except IndexError:\n",
    "            return 0\n",
    "        else:\n",
    "          return 0\n",
    "    ######################################################################\n",
    "    def friendInfluence(self, userId):\n",
    "        \"\"\"\n",
    "        朋友对用户的影响\n",
    "        主要考虑用户所有的朋友中，有多少是非常喜欢参加各种社交活动/event的\n",
    "        用户的朋友圈如果都积极参与各种event，可能会对当前用户有一定的影响\n",
    "        \"\"\"\n",
    "        nusers = np.shape(self.userFriends)[1]\n",
    "        i = self.userIndex[userId]\n",
    "        return (self.userFriends[i, :].sum(axis=0) / nusers)[0,0]\n",
    "\n",
    "    def eventPop(self, eventId):\n",
    "        \"\"\"\n",
    "        本活动本身的热度\n",
    "        主要是通过参与的人数来界定的\n",
    "        \"\"\"\n",
    "        i = self.eventIndex[eventId]\n",
    "        return self.eventPopularity[i, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRSData(RS, train=True, header=True):\n",
    "    \"\"\"\n",
    "    把前面user-based协同过滤 和 item-based协同过滤，以及各种热度和影响度作为特征组合在一起\n",
    "    生成新的训练数据，用于分类器分类使用\n",
    "    \"\"\"\n",
    "    fn = \"train.csv\" if train else \"test.csv\"\n",
    "    #fin = open(fn, 'rb')\n",
    "    fin = open(fn, 'r')\n",
    "    #fout = open(\"RS_\" + fn, 'wb')\n",
    "    fout = open(\"RS_\" + fn, 'w')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    fin.readline().strip().split(\",\")\n",
    "    \n",
    "    # write output header\n",
    "    if header:\n",
    "      ocolnames = [\"invited\", \"userCF_reco\", \"evtCF_reco\",\"svdCF_reco\",\"user_reco\", \"evt_p_reco\",\n",
    "        \"evt_c_reco\", \"user_pop\", \"frnd_infl\", \"evt_pop\"]\n",
    "      if train:\n",
    "        ocolnames.append(\"interested\")\n",
    "        ocolnames.append(\"not_interested\")\n",
    "      fout.write(\",\".join(ocolnames) + \"\\n\")\n",
    "    \n",
    "    ln = 0\n",
    "    for line in fin:\n",
    "      ln += 1\n",
    "      if ln%500 == 0:\n",
    "          print (\"%s:%d (userId, eventId)=(%s, %s)\" % (fn, ln, userId, eventId))\n",
    "          #break;\n",
    "      \n",
    "      cols = line.strip().split(\",\")\n",
    "      userId = cols[0]\n",
    "      eventId = cols[1]\n",
    "      invited = cols[2]\n",
    "      \n",
    "      userCF_reco = RS.userCFReco(userId, eventId)\n",
    "      itemCF_reco = RS.eventCFReco(userId, eventId)\n",
    "      svdCF_reco = RS.svdCFReco(userId, eventId)\n",
    "        \n",
    "      user_reco = RS.userReco(userId, eventId)\n",
    "      evt_p_reco, evt_c_reco = RS.eventReco(userId, eventId)\n",
    "      user_pop = RS.userPop(userId)\n",
    "     \n",
    "      frnd_infl = RS.friendInfluence(userId)\n",
    "      evt_pop = RS.eventPop(eventId)\n",
    "      ocols = [invited, userCF_reco, itemCF_reco, svdCF_reco,user_reco, evt_p_reco,\n",
    "        evt_c_reco, user_pop, frnd_infl, evt_pop]\n",
    "      \n",
    "      if train:\n",
    "        ocols.append(cols[4]) # interested\n",
    "        ocols.append(cols[5]) # not_interested\n",
    "      fout.write(\",\".join(map(lambda x: str(x), ocols)) + \"\\n\")\n",
    "    \n",
    "    fin.close()\n",
    "    fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RS = RecomSys()\n",
    "# print (\"生成训练数据...\\n\")\n",
    "# generateRSData(RS,train=True,  header=True)\n",
    "\n",
    "# print (\"生成预测数据...\\n\")\n",
    "# generateRSData(RS, train=False, header=True)\n",
    "\n",
    "#会出现MemoryError:  计算机提示内存不足"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
